import urllib.request
import re
import requests

#设置爬取参数
set_url = "https://maoyan.com/board/4?offset="
target_url_list=[]
for target_number in range(0,10):
	target_url=set_url+str(target_number*10)
	target_url_list.append(target_url)
        #print(target_url_list)
	ranking ='<i class="board-index board-index-.*?">(.*?)</i>'
	image = ' <img data-src="(.*?)" alt=".*?" class="board-img" />'
	title = '<a href=.*? title=.*? data-act="boarditem-click" data-val=.*?>(.*?)</a>'
	#star = '主演：(.*?)[]{8}'
	#star = '<p class="star">(.*?)</p>'
	star = '主演：(.*?)\n'
	releasetime = '<p class="releasetime">(.*?)</p>'
	score_integer = '<i class="integer">(.*?)</i>'
	score_fraction = '<i class="fraction">(.*?)</i>'

#创建空列表，用来存储信息	
ranking_list=[]
image_list=[]
title_list=[]
star_list=[]
releasetime_list=[]
score_integer_list=[]
score_fraction_list=[]

#开始爬取
for the_url in target_url_list:
	url = the_url
	req = urllib.request.Request(url)
	res = urllib.request.urlopen(req)
	html = res.read().decode('utf-8')
	ranking_list.append(re.findall(ranking,html))
	image_list.append(re.findall(image,html))
	title_list.append(re.findall(title,html))
	star_list.append(re.findall(star,html,re.S))
	releasetime_list.append(re.findall(releasetime,html))
	score_integer_list.append(re.findall(score_integer,html))
	score_fraction_list.append(re.findall(score_fraction,html))

#print(ranking_list)
#print(image_list)
#print(title_list)
#print(star_list)
#print(releasetime_list)
#print(score_integer_list)
#print(score_fraction_list)

#将每一页的表汇总到大表上
total_ranking_list=[]
total_image_list=[]
total_title_list=[]
total_star_list=[]
total_releasetime_list=[]
total_score_integer_list=[]
total_score_fraction_list=[]

for firstlever in range(0,10):
	for secondlever in range(0,10):
		total_ranking_list.append(ranking_list[firstlever][secondlever])
		total_image_list.append(image_list[firstlever][secondlever])
		total_title_list.append(title_list[firstlever][secondlever])
		total_star_list.append(star_list[firstlever][secondlever])
		total_releasetime_list.append(releasetime_list[firstlever][secondlever])
		total_score_integer_list.append(score_integer_list[firstlever][secondlever])
		total_score_fraction_list.append(score_fraction_list[firstlever][secondlever])

#猫眼的分数分为整数部分与小数部分，需要整理。
total_score_list=[]
for firstlever in range(0,100):
	total_score_list.append(float(str(total_score_integer_list[firstlever])+str(total_score_fraction_list[firstlever])))

#开始整理总表
total_information_list_of_a_movie=[]
information_of_a_movie=[]

# for secondlever in range(0,100):
# 	information_of_a_movie[0]=total_ranking_list[secondlever]
# 	information_of_a_movie[1]=total_image_list[secondlever]
# 	information_of_a_movie[2]=total_title_list[secondlever]
# 	information_of_a_movie[3]=total_star_list[secondlever]
# 	information_of_a_movie[4]=total_releasetime_list[secondlever]
# 	information_of_a_movie[5]=total_score_list[secondlever]
# 	total_information_list_of_a_movie.append(information_of_a_movie)

for secondlever in range(0,100):
	information_of_a_movie.append(total_ranking_list[secondlever])
	information_of_a_movie.append(total_image_list[secondlever])
	information_of_a_movie.append(total_title_list[secondlever])
	information_of_a_movie.append(total_star_list[secondlever])
	information_of_a_movie.append(total_releasetime_list[secondlever])
	information_of_a_movie.append(total_score_list[secondlever])
	total_information_list_of_a_movie.append(information_of_a_movie)
	information_of_a_movie=[]


#print(total_information_list_of_a_movie)

#写入文件
filename = 'movie_data.txt'
with open(filename,'w') as f:
        f.write(str(total_information_list_of_a_movie))
        f.close()

print("爬取以及写入完毕")





	


	



